1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32 import java.util.*;
33 import java.util.regex.*;
34 import java.nio.*;
35 import java.nio.charset.*;
36
37 public class FindDecoderBugs {
38
39 static boolean isBroken(String csn) {
40 if (csn.equals("x-COMPOUND_TEXT")) return true;
41 return false;
42 }
43
44 static <T extends Comparable<? super T>> List<T> sort(Collection<T> c) {
45 List<T> list = new ArrayList<T>(c);
46 Collections.sort(list);
47 return list;
48 }
49
50 static class TooManyFailures extends RuntimeException {
51 private static final long serialVersionUID = 0L;
52 }
53
54 static String string(byte[] a) {
55 final StringBuilder sb = new StringBuilder();
56 for (byte b : a) {
57 if (sb.length() != 0) sb.append(' ');
58 sb.append(String.format("%02x", b & 0xff));
59 }
60 return sb.toString();
61 }
62
63 static String string(char[] a) {
64 final StringBuilder sb = new StringBuilder();
65 for (char c : a) {
66 if (sb.length() != 0) sb.append(' ');
67 sb.append(String.format("\\u%04x", (int) c));
68 }
69 return sb.toString();
70 }
71
72 static class Reporter {
73
74
75 static class Counts extends HashMap<String, Long> {
76 private static final long serialVersionUID = -1;
77 long inc(String signature) {
78 Long count = get(signature);
79 if (count == null) count = 0L;
80 put(signature, count+1);
81 return count+1;
82 }
83 }
84
85 final Counts failureCounts = new Counts();
86 final static long maxFailures = 2;
87
88 final static Pattern hideBytes = Pattern.compile("\"[0-9a-f ]+\"");
89 final static Pattern hideChars = Pattern.compile("\\\\u[0-9a-f]{4}");
90
91 boolean bug(String format, Object... args) {
92 String signature = String.format(format, args);
93 signature = hideBytes.matcher(signature).replaceAll("\"??\"");
94 signature = hideChars.matcher(signature).replaceAll("\\u????");
95 failed++;
96 if (failureCounts.inc(signature) <= maxFailures) {
97 System.out.printf(format, args);
98 System.out.println();
99 return true;
100 }
101 return false;
102 }
103
104 void summarize() {
105 for (String key : sort(failureCounts.keySet()))
106 System.out.printf("-----%n%s%nfailures=%d%n",
107 key, failureCounts.get(key));
108 }
109 }
110
111 static final Reporter reporter = new Reporter();
112
113 static class Result {
114 final int limit;
115 final int ipos;
116 final boolean direct;
117 final byte[] ia;
118 final char[] oa;
119 final CoderResult cr;
120
121 Result(ByteBuffer ib, CharBuffer ob, CoderResult cr) {
122 ipos = ib.position();
123 ia = toArray(ib);
124 oa = toArray(ob);
125 direct = ib.isDirect();
126 limit = ob.limit();
127 this.cr = cr;
128 }
129
130 static byte[] toArray(ByteBuffer b) {
131 int pos = b.position();
132 byte[] a = new byte[b.limit()];
133 b.position(0);
134 b.get(a);
135 b.position(pos);
136 return a;
137 }
138
139 static char[] toArray(CharBuffer b) {
140 char[] a = new char[b.position()];
141 b.position(0);
142 b.get(a);
143 return a;
144 }
145
146 static boolean eq(Result x, Result y) {
147 return x == y ||
148 (x != null && y != null &&
149 (Arrays.equals(x.oa, y.oa) &&
150 x.ipos == y.ipos &&
151 x.cr == y.cr));
152 }
153
154 public String toString() {
155 return String.format("\"%s\"[%d/%d] => %s \"%s\"[%d/%d]%s",
156 string(ia), ipos, ia.length,
157 cr, string(oa), oa.length, limit,
158 (direct ? " (direct)" : ""));
159 }
160 }
161
162
163 static final int maxBufSize = 20;
164 static final ByteBuffer[] ribs = new ByteBuffer[maxBufSize];
165 static final ByteBuffer[] dibs = new ByteBuffer[maxBufSize];
166
167 static final CharBuffer[] robs = new CharBuffer[maxBufSize];
168 static final CharBuffer[] dobs = new CharBuffer[maxBufSize];
169 static {
170 for (int i = 0; i < maxBufSize; i++) {
171 ribs[i] = ByteBuffer.allocate(i);
172 dibs[i] = ByteBuffer.allocateDirect(i);
173 robs[i] = CharBuffer.allocate(i);
174 dobs[i] = ByteBuffer.allocateDirect(i*2).asCharBuffer();
175 }
176 }
177
178 static class CharsetTester {
179 private final Charset cs;
180 private static final long maxFailures = 5;
181 private long failures = 0;
182
183 private static final long maxCharsetFailures = 10000L;
184 private final long failed0 = failed;
185
186 CharsetTester(Charset cs) {
187 this.cs = cs;
188 }
189
190 static boolean bug(String format, Object... args) {
191 return reporter.bug(format, args);
192 }
193
194 Result recode(ByteBuffer ib, CharBuffer ob) {
195 try {
196 char canary = '\u4242';
197 ib.clear();
198 ob.clear();
199 for (int i = 0; i < ob.limit(); i++)
200 ob.put(i, canary);
201 CharsetDecoder coder = cs.newDecoder();
202 CoderResult cr = coder.decode(ib, ob, false);
203 equal(ib.limit(), ib.capacity());
204 equal(ob.limit(), ob.capacity());
205 Result r = new Result(ib, ob, cr);
206 if (cr.isError())
207 check(cr.length() > 0);
208 if (cr.isOverflow() && ob.remaining() > 10)
209 bug("OVERFLOW, but there's lots of room: %s %s",
210 cs, r);
211
212
213
214 if (cr.isError() && ib.remaining() < cr.length())
215 bug("remaining() < CoderResult.length(): %s %s",
216 cs, r);
217
218
219
220
221 CoderResult cr2 = coder.decode(ib, ob, false);
222 if (ib.position() != r.ipos ||
223 ob.position() != r.oa.length ||
224 cr != cr2)
225 bug("Coding operation not idempotent: %s%n %s%n %s",
226 cs, r, new Result(ib, ob, cr2));
227 if (ob.position() < ob.limit() &&
228 ob.get(ob.position()) != canary)
229 bug("Buffer overrun: %s %s %s",
230 cs, r, ob.get(ob.position()));
231 return r;
232 } catch (Throwable t) {
233 if (bug("Unexpected exception: %s %s %s",
234 cs, t.getClass().getSimpleName(),
235 new Result(ib, ob, null)))
236 t.printStackTrace();
237 return null;
238 }
239 }
240
241 Result recode2(byte[] ia, int n) {
242 int len = ia.length;
243 ByteBuffer rib = ByteBuffer.wrap(ia);
244 ByteBuffer dib = dibs[len];
245 dib.clear(); dib.put(ia); dib.clear();
246 CharBuffer rob = robs[n];
247 CharBuffer dob = dobs[n];
248 equal(rob.limit(), n);
249 equal(dob.limit(), n);
250 check(dib.isDirect());
251 check(dob.isDirect());
252 Result r1 = recode(rib, rob);
253 Result r2 = recode(dib, dob);
254 if (r1 != null && r2 != null && ! Result.eq(r1, r2))
255 bug("Results differ for direct buffers: %s%n %s%n %s",
256 cs, r1, r2);
257 return r1;
258 }
259
260 Result test(byte[] ia) {
261 if (failed - failed0 >= maxCharsetFailures)
262 throw new TooManyFailures();
263
264 Result roomy = recode2(ia, maxBufSize - 1);
265 if (roomy == null) return roomy;
266 int olen = roomy.oa.length;
267 if (olen > 0) {
268 if (roomy.ipos == roomy.ia.length) {
269 Result perfectFit = recode2(ia, olen);
270 if (! Result.eq(roomy, perfectFit))
271 bug("Results differ: %s%n %s%n %s",
272 cs, roomy, perfectFit);
273 }
274 for (int i = 0; i < olen; i++) {
275 Result claustrophobic = recode2(ia, i);
276 if (claustrophobic == null) return roomy;
277 if (roomy.cr.isUnderflow() &&
278 ! claustrophobic.cr.isOverflow())
279 bug("Expected OVERFLOW: %s%n %s%n %s",
280 cs, roomy, claustrophobic);
281 }
282 }
283 return roomy;
284 }
285
286 void testExhaustively(byte[] prefix, int n) {
287 int len = prefix.length;
288 byte[] ia = Arrays.copyOf(prefix, len + 1);
289 for (int i = 0; i < 0x100; i++) {
290 ia[len] = (byte) i;
291 if (n == 1)
292 test(ia);
293 else
294 testExhaustively(ia, n - 1);
295 }
296 }
297
298 void testRandomly(byte[] prefix, int n) {
299 int len = prefix.length;
300 byte[] ia = Arrays.copyOf(prefix, len + n);
301 for (int i = 0; i < 5000; i++) {
302 for (int j = 0; j < n; j++)
303 ia[len + j] = randomByte();
304 test(ia);
305 }
306 }
307
308 void testPrefix(byte[] prefix) {
309 if (prefix.length > 0)
310 System.out.printf("Testing prefix %s%n", string(prefix));
311
312 test(prefix);
313
314 testExhaustively(prefix, 1);
315 testExhaustively(prefix, 2);
316
317
318
319 testRandomly(prefix, 3);
320 testRandomly(prefix, 4);
321 }
322 }
323
324 private final static Random rnd = new Random();
325 private static byte randomByte() {
326 return (byte) rnd.nextInt(0x100);
327 }
328 private static byte[] randomBytes(int len) {
329 byte[] a = new byte[len];
330 for (int i = 0; i < len; i++)
331 a[i] = randomByte();
332 return a;
333 }
334
335 private static final byte SS2 = (byte) 0x8e;
336 private static final byte SS3 = (byte) 0x8f;
337 private static final byte ESC = (byte) 0x1b;
338 private static final byte SO = (byte) 0x0e;
339 private static final byte SI = (byte) 0x0f;
340
341 private final static byte[][] stateChangers = {
342 {SS2}, {SS3}, {SO}, {SI}
343 };
344
345 private final static byte[][]escapeSequences = {
346 {ESC, '(', 'B'},
347 {ESC, '(', 'I'},
348 {ESC, '(', 'J'},
349 {ESC, '$', '@'},
350 {ESC, '$', 'A'},
351 {ESC, '$', ')', 'A'},
352 {ESC, '$', ')', 'C'},
353 {ESC, '$', ')', 'G'},
354 {ESC, '$', '*', 'H'},
355 {ESC, '$', '+', 'I'},
356 {ESC, '$', 'B'},
357 {ESC, 'N'},
358 {ESC, 'O'},
359 {ESC, '$', '(', 'D'},
360 };
361
362 private static boolean isStateChanger(Charset cs, byte[] ia) {
363 Result r = new CharsetTester(cs).recode2(ia, 9);
364 return r == null ? false :
365 (r.cr.isUnderflow() &&
366 r.ipos == ia.length &&
367 r.oa.length == 0);
368 }
369
370 private final static byte[][] incompletePrefixes = {
371 {ESC},
372 {ESC, '('},
373 {ESC, '$'},
374 {ESC, '$', '(',},
375 };
376
377 private static boolean isIncompletePrefix(Charset cs, byte[] ia) {
378 Result r = new CharsetTester(cs).recode2(ia, 9);
379 return r == null ? false :
380 (r.cr.isUnderflow() &&
381 r.ipos == 0 &&
382 r.oa.length == 0);
383 }
384
385 private static void testCharset(Charset cs) throws Throwable {
386 final String csn = cs.name();
387
388 if (isBroken(csn)) {
389 System.out.printf("Skipping possibly broken charset %s%n", csn);
390 return;
391 }
392 System.out.println(csn);
393 CharsetTester tester = new CharsetTester(cs);
394
395 tester.testPrefix(new byte[0]);
396
397 if (! csn.matches("(?:x-)?(?:UTF|JIS(?:_X)?0).*")) {
398 for (byte[] prefix : stateChangers)
399 if (isStateChanger(cs, prefix))
400 tester.testPrefix(prefix);
401
402 for (byte[] prefix : incompletePrefixes)
403 if (isIncompletePrefix(cs, prefix))
404 tester.testPrefix(prefix);
405
406 if (isIncompletePrefix(cs, new byte[] {ESC}))
407 for (byte[] prefix : escapeSequences)
408 if (isStateChanger(cs, prefix))
409 tester.testPrefix(prefix);
410 }
411 }
412
413 private static void realMain(String[] args) {
414 for (Charset cs : sort(Charset.availableCharsets().values())) {
415 try {
416 testCharset(cs);
417 } catch (TooManyFailures e) {
418 System.out.printf("Too many failures for %s%n", cs);
419 } catch (Throwable t) {
420 unexpected(t);
421 }
422 }
423 reporter.summarize();
424 }
425
426
427 static volatile long passed = 0, failed = 0;
428 static void pass() {passed++;}
429 static void fail() {failed++; Thread.dumpStack();}
430 static void fail(String format, Object... args) {
431 System.out.println(String.format(format, args)); failed++;}
432 static void fail(String msg) {System.out.println(msg); fail();}
433 static void unexpected(Throwable t) {failed++; t.printStackTrace();}
434 static void check(boolean cond) {if (cond) pass(); else fail();}
435 static void equal(Object x, Object y) {
436 if (x == null ? y == null : x.equals(y)) pass();
437 else fail(x + " not equal to " + y);}
438 static void equal(int x, int y) {
439 if (x == y) pass();
440 else fail(x + " not equal to " + y);}
441 public static void main(String[] args) throws Throwable {
442 try {realMain(args);} catch (Throwable t) {unexpected(t);}
443 System.out.printf("%nPassed = %d, failed = %d%n%n", passed, failed);
444 if (failed > 0) throw new AssertionError("Some tests failed");}
445 }